Data is from August, 2017 sequencing. This is week 4 of experiment 1

Download and map:

Working on Whitney for read mapping. Directory: /Network/Servers/avalanche.plb.ucdavis.edu/Volumes/Mammoth/Users/jmaloof/2017/Wyoming-microbiome/20170830-data/

20170830 reads

Build index (not done…already exists)

cd ~/Sequences/ref_genomes/B_rapa/genome/V3.0

wget http://brassicadb.org/brad/datasets/pub/BrassicaceaeGenome/Brassica_rapa/V3.0/Brapa_genome_v3.0_cds.fasta.gz 

kallisto index -i B_rapa_CDS_V3.0_k31_kallisto_index Brapa_genome_v3.0_cds.fasta.gz

cd ~/2017/Wyoming-microbiome//20170830-samples/20170830-data

Map reads

mkdir kallisto_out_V3.0

#actually a fish loop
for file in (ls raw-fastq/2017-08-27/*.fastq.gz)
  echo $file
  set newfile (basename $file _R1_001.fastq.gz)
  kallisto quant -i ~/Sequences/ref_genomes/B_rapa/genome/V3.0/B_rapa_CDS_V3.0_k31_kallisto_index  -o kallisto_out_V3.0/$newfile --single -l 200 -s 40 -t 4 --plaintext $file
end

Move the counts to my local computer

NEED TO MODIFY BELOW

cd /Users/jmaloof/git/Br_Microbe_Paper_2021/RNA/input/20170830-samples
lftp sftp://whitney.plb.ucdavis.edu
  cd 2017/Wyoming-microbiome/20170830-samples/20170830-data
  mirror kallisto_out_V3.0

remove unused files

cd /Users/jmaloof/git/Br_Microbe_Paper_2021/RNA/input/20170830-samples/kallisto_out_V3.0
rm */*.json

compress tsv files

cd /Users/jmaloof/git/Br_Microbe_Paper_2021/RNA/input/20170830-samples/kallisto_out_V3.0
gzip */abundance.tsv

Get counts into R

library(tidyverse)
Registered S3 methods overwritten by 'dbplyr':
  method         from
  print.tbl_lazy     
  print.tbl_sql      
── Attaching packages ──────────────────────────────── tidyverse 1.3.1 ──
✓ ggplot2 3.3.3     ✓ purrr   0.3.4
✓ tibble  3.1.2     ✓ dplyr   1.0.6
✓ tidyr   1.1.3     ✓ stringr 1.4.0
✓ readr   1.4.0     ✓ forcats 0.5.1
── Conflicts ─────────────────────────────────── tidyverse_conflicts() ──
x dplyr::filter() masks stats::filter()
x dplyr::lag()    masks stats::lag()
library(stringr)
library(edgeR)
Loading required package: limma
head(kallisto_names)
[1] "wyo_leaf_R500_10_417_S65_L001" "wyo_leaf_R500_10_417_S65_L002"
[3] "wyo_leaf_R500_10_417_S65_L003" "wyo_leaf_R500_10_417_S65_L004"
[5] "wyo_leaf_R500_10_419_S66_L001" "wyo_leaf_R500_10_419_S66_L002"
counts <- tibble(sample = kallisto_names, file = kallisto_files) %>%
  mutate(countdata = map(kallisto_files, read_tsv))

── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)
head(counts)

reformat into rows = genes and columns = samples

counts <- counts %>% unnest(countdata) %>%
  mutate(sample = str_remove(sample, "_L.*")) %>% 
  select(sample, target_id, est_counts) %>%
  group_by(sample, target_id) %>%
  summarize(est_counts=sum(est_counts)) %>% # sum up counts from multiple lanes
  ungroup() %>%
  pivot_wider(id_cols = target_id,
              names_from = sample,
              values_from = est_counts)
`summarise()` has grouped output by 'sample'. You can override using the `.groups` argument.
head(counts)
dim(counts)
[1] 46250    25
write_csv(counts,"../../output/20170830_V3.0_raw_counts_.csv.gz")

make sample description data frame

sample.description <- tibble(sample=colnames(counts)[-1]) %>%
  separate(sample,
           c("location","tissue","genotype","block","pot"),
           remove=FALSE,
           convert=TRUE) 
head(sample.description)

## get additional metadata
sample.info <- readxl::read_excel("../input/wy001_RNAseq.xlsx",sheet = 1)
head(sample.info)

##combine
sample.description <- left_join(sample.description, sample.info)
sample.description <- sample.description %>% 
  mutate(group=paste(tissue,genotype,soil,autoclave,sep="_"))
head(sample.description)
sample.description %>% summarize(n_distinct(group))

summarize counts

pl.orig <- counts[,-1] %>% colSums() %>% tibble(sample=names(.),count=.) %>%
  ggplot(aes(x=sample,y=count)) + 
  geom_col() +
  theme(axis.text.x  = element_text(angle=90, vjust=0.5,size = 7)) 
pl.orig

load to edgeR, normalize

#confirm that everthing is in the right order
all(colnames(counts)[-1]==sample.description$sample)
dge <- DGEList(counts[,-1],
               group=sample.description$group,
               samples=sample.description,
               genes=counts$target_id)
dge <- calcNormFactors(dge)
barplot(dge$samples$lib.size)
ggplot(dge$samples,aes(x=sample,y=norm.factors,fill=tissue)) + geom_col() + 
  theme(axis.text.x  = element_text(angle=90, vjust=0.5,size = 7)) 
ggplot(dge$samples,aes(x=sample,y=norm.factors,fill=genotype)) + geom_col() + 
  theme(axis.text.x  = element_text(angle=90, vjust=0.5,size = 7)) 
ggplot(dge$samples,aes(x=sample,y=norm.factors,fill=as.factor(block))) + geom_col() +
  theme(axis.text.x  = element_text(angle=90, vjust=0.5,size = 7)) 

Looks like we should normalize separately for root and leaf

do separately for leaf and root values

counts.leaf <- counts %>% select(target_id, contains("leaf"))
counts.root <- counts %>% select(target_id, contains("root"))
sample.description.leaf <- sample.description %>% filter(tissue=="leaf")
sample.description.root <- sample.description %>% filter(tissue=="root")

Leaf

#confirm that everthing is in the right order
all(colnames(counts.leaf)[-1]==sample.description.leaf$sample)
dge.leaf <- DGEList(counts.leaf[,-1],
                    group=sample.description.leaf$group,
                    samples=sample.description.leaf,
                    genes=counts.leaf$target_id)
dge.leaf <- calcNormFactors(dge.leaf)

Root

#confirm that everthing is in the right order
all(colnames(counts.root)[-1]==sample.description.root$sample)
dge.root <- DGEList(counts.root[,-1],
                    group=sample.description.root$group,
                    samples=sample.description.root,
                    genes=counts.root$target_id)
dge.root <- calcNormFactors(dge.root)
save(dge.leaf,dge.root,sample.description.leaf,sample.description.root,file="../output/edgeR_dge_objects.Rdata")

Write out cpm values

cpm.leaf.w <- bind_cols(dge.leaf$gene,as_tibble(cpm(dge.leaf))) %>% as_tibble() %>% rename(transcript_ID=genes)
head(cpm.leaf.w)
write_csv(cpm.leaf.w,"../output/cpm_wide_20170617_leaf_samples.csv.gz")
cpm.root.w <- bind_cols(dge.root$gene,as_tibble(cpm(dge.root))) %>% as_tibble() %>% rename(transcript_ID=genes)
head(cpm.root.w)
write_csv(cpm.root.w,"../output/cpm_wide_20170617_root_samples.csv.gz")

Also let’s reformat this to long format and add metadata

cpm.leaf.long <- cpm.leaf.w %>% 
  gather(-transcript_ID,key = sample,value=cpm) %>%
  left_join(sample.description.leaf)
head(cpm.leaf.long)
write_csv(cpm.leaf.long,"../output/cpm_long_with_metadata_20170617_leaf_samples.csv.gz")
cpm.root.long <- cpm.root.w %>% 
  gather(-transcript_ID,key = sample,value=cpm) %>%
  left_join(sample.description.root)
head(cpm.root.long)
write_csv(cpm.root.long,"../output/cpm_long_with_metadata_20170617_root_samples.csv.gz")

compute and write out voom-transformed values

design.leaf <- model.matrix(~ sample.description.leaf$group)
dge4voom.leaf <- dge.leaf[rowSums(cpm(dge.leaf)>1) >= 6,,keep.lib.sizes=FALSE]
dge4voom.leaf <- calcNormFactors(dge4voom.leaf)
data.voom.leaf <- voom(dge4voom.leaf,design = design.leaf)
data.voom.exp.leaf <- bind_cols(data.voom.leaf$genes,as_tibble(data.voom.leaf$E)) %>%
  rename(transcript_ID=genes) %>% as_tibble()
head(data.voom.exp.leaf)
write_csv(data.voom.exp.leaf, "../output/voom_expression_20170617_T6_leaf_samples.csv.gz")
voom.long.leaf <- data.voom.exp.leaf %>% 
  gather(-transcript_ID,key = sample,value=expression) %>%
  left_join(sample.description.leaf)
head(voom.long.leaf)
hist(voom.long.leaf$expression)
write_csv(voom.long.leaf,"../output/voom_long_with_metadata_20170617_T6_leaf_samples.csv.gz")
design.root <- model.matrix(~ sample.description.root$group)
dge4voom.root <- dge.root[rowSums(cpm(dge.root)>1) >= 6,,keep.lib.sizes=FALSE]
dge4voom.root <- calcNormFactors(dge4voom.root)
data.voom.root <- voom(dge4voom.root,design = design.root)
data.voom.exp.root <- bind_cols(data.voom.root$genes,as_tibble(data.voom.root$E)) %>%
  rename(transcript_ID=genes) %>% as_tibble()
head(data.voom.exp.root)
write_csv(data.voom.exp.root, "../output/voom_expression_20170617_T6_root_samples.csv.gz")
voom.long.root <- data.voom.exp.root %>% 
  gather(-transcript_ID,key = sample,value=expression) %>%
  left_join(sample.description.root)
head(voom.long.root)
hist(voom.long.root$expression)
write_csv(voom.long.root,"../output/voom_long_with_metadata_20170617_T6_root_samples.csv.gz")

write it to irods

Need to run this yourself in terminal

iinit
icd /iplant/home/shared/ucd.brassica/analyses/Brapa_Microbiome_RNAseq/
for f in (ls cpm*)
    echo $f
    iput -vf $f
end
for f in (ls voom*)
    echo $f
    iput -vf $f
end

read distribution

Mike asked if the difference in normalization factors in leafs vs roots was due to high abundance of photosynthesis transcripts in leafs. (Although leafs have lower normalization factor)

Start by doing a simplistic normalization just by library size. Then look at distribution of most abundant counts.

counts.leaf.norm <- counts.leaf %>% 
  mutate_at(-1,funs(./sum(.))) %>% 
  transmute(target_id=target_id,mean= {select(.,-target_id) %>% rowMeans()}) %>%
  arrange(desc(mean)) %>% mutate(cumsum=cumsum(mean),rank=row_number(),sample="leaf")
counts.leaf.norm
counts.root.norm <- counts.root %>% 
  mutate_at(-1,funs(./sum(.))) %>% 
  transmute(target_id=target_id,mean= {select(.,-target_id) %>% rowMeans()}) %>%
  arrange(desc(mean)) %>% mutate(cumsum=cumsum(mean),rank=row_number(),sample="root")
counts.root.norm
rbind(counts.leaf.norm,counts.root.norm) %>%
  ggplot(aes(x=rank,y=cumsum,color=sample)) +
  geom_line() +xlim(0,20000)
rbind(counts.leaf.norm,counts.root.norm) %>%
  ggplot(aes(x=rank,y=mean,color=sample)) +
  geom_line()  + scale_y_log10()
rbind(counts.leaf.norm,counts.root.norm) %>% filter(rank < 41) %>%
  ggplot(aes(x=rank,y=mean,fill=sample)) +
  geom_col(position = "dodge") 
annotation <- read_csv("../../../Annotation/output/v3.0annotation/Brapa_V3.0_annotated.csv")
top.expressed.leaf <- counts.leaf.norm %>% filter(rank<21) %>% left_join(annotation,by=c("target_id"="name")) %>% select("target_id","mean","rank","AGI","At_symbol","At_description") %>% arrange(rank)
top.expressed.leaf
write_csv(top.expressed.leaf,"../output/top.expressed.leaf.csv")
top.expressed.root <- counts.root.norm %>% filter(rank<21) %>% left_join(annotation,by=c("target_id"="name")) %>% select("target_id","mean","rank","AGI","At_symbol","At_description") %>% arrange(rank)
top.expressed.root
write_csv(top.expressed.root,"../output/top.expressed.root.csv")
LS0tCnRpdGxlOiAiUk5BIGV4cHJlc3Npb24gYW5hbHlzaXMgb2YgQnJhc3NpY2EgTWljcm9iZSBEYXRhLiBJOiBwcmVwIGRhdGEiCm91dHB1dDogaHRtbF9ub3RlYm9vawotLS0KCkRhdGEgaXMgZnJvbSBBdWd1c3QsIDIwMTcgc2VxdWVuY2luZy4gIFRoaXMgaXMgd2VlayA0IG9mIGV4cGVyaW1lbnQgMQoKIyMgRG93bmxvYWQgYW5kIG1hcDoKCldvcmtpbmcgb24gV2hpdG5leSBmb3IgcmVhZCBtYXBwaW5nLiAgRGlyZWN0b3J5OiBgL05ldHdvcmsvU2VydmVycy9hdmFsYW5jaGUucGxiLnVjZGF2aXMuZWR1L1ZvbHVtZXMvTWFtbW90aC9Vc2Vycy9qbWFsb29mLzIwMTcvV3lvbWluZy1taWNyb2Jpb21lLzIwMTcwODMwLWRhdGEvYAoKIyMjIDIwMTcwODMwIHJlYWRzCgpCdWlsZCBpbmRleCAobm90IGRvbmUuLi5hbHJlYWR5IGV4aXN0cykKYGBge3IsIGVuZ2luZT0nYmFzaCcsIGV2YWw9RkFMU0V9CmNkIH4vU2VxdWVuY2VzL3JlZl9nZW5vbWVzL0JfcmFwYS9nZW5vbWUvVjMuMAoKd2dldCBodHRwOi8vYnJhc3NpY2FkYi5vcmcvYnJhZC9kYXRhc2V0cy9wdWIvQnJhc3NpY2FjZWFlR2Vub21lL0JyYXNzaWNhX3JhcGEvVjMuMC9CcmFwYV9nZW5vbWVfdjMuMF9jZHMuZmFzdGEuZ3ogCgprYWxsaXN0byBpbmRleCAtaSBCX3JhcGFfQ0RTX1YzLjBfazMxX2thbGxpc3RvX2luZGV4IEJyYXBhX2dlbm9tZV92My4wX2Nkcy5mYXN0YS5negoKY2Qgfi8yMDE3L1d5b21pbmctbWljcm9iaW9tZS8vMjAxNzA4MzAtc2FtcGxlcy8yMDE3MDgzMC1kYXRhCmBgYAoKTWFwIHJlYWRzCgpgYGB7ciwgZW5naW5lPSdiYXNoJyxldmFsPUZBTFNFfQpta2RpciBrYWxsaXN0b19vdXRfVjMuMAoKI2FjdHVhbGx5IGEgZmlzaCBsb29wCmZvciBmaWxlIGluIChscyByYXctZmFzdHEvMjAxNy0wOC0yNy8qLmZhc3RxLmd6KQogIGVjaG8gJGZpbGUKICBzZXQgbmV3ZmlsZSAoYmFzZW5hbWUgJGZpbGUgX1IxXzAwMS5mYXN0cS5neikKICBrYWxsaXN0byBxdWFudCAtaSB+L1NlcXVlbmNlcy9yZWZfZ2Vub21lcy9CX3JhcGEvZ2Vub21lL1YzLjAvQl9yYXBhX0NEU19WMy4wX2szMV9rYWxsaXN0b19pbmRleCAgLW8ga2FsbGlzdG9fb3V0X1YzLjAvJG5ld2ZpbGUgLS1zaW5nbGUgLWwgMjAwIC1zIDQwIC10IDQgLS1wbGFpbnRleHQgJGZpbGUKZW5kCmBgYAoKTW92ZSB0aGUgY291bnRzIHRvIG15IGxvY2FsIGNvbXB1dGVyCgojIyMgTkVFRCBUTyBNT0RJRlkgQkVMT1cKCmBgYHtyLCBlbmdpbmU9J2Jhc2gnLCBldmFsPUZBTFNFfQpjZCAvVXNlcnMvam1hbG9vZi9naXQvQnJfTWljcm9iZV9QYXBlcl8yMDIxL1JOQS9pbnB1dC8yMDE3MDgzMC1zYW1wbGVzCmxmdHAgc2Z0cDovL3doaXRuZXkucGxiLnVjZGF2aXMuZWR1CiAgY2QgMjAxNy9XeW9taW5nLW1pY3JvYmlvbWUvMjAxNzA4MzAtc2FtcGxlcy8yMDE3MDgzMC1kYXRhCiAgbWlycm9yIGthbGxpc3RvX291dF9WMy4wCmBgYAoKcmVtb3ZlIHVudXNlZCBmaWxlcwpgYGB7ciwgZW5naW5lPSdiYXNoJywgZXZhbD1GQUxTRX0KY2QgL1VzZXJzL2ptYWxvb2YvZ2l0L0JyX01pY3JvYmVfUGFwZXJfMjAyMS9STkEvaW5wdXQvMjAxNzA4MzAtc2FtcGxlcy9rYWxsaXN0b19vdXRfVjMuMApybSAqLyouanNvbgpgYGAKCmNvbXByZXNzIHRzdiBmaWxlcwpgYGB7ciwgZW5naW5lPSdiYXNoJywgZXZhbD1GQUxTRX0KY2QgL1VzZXJzL2ptYWxvb2YvZ2l0L0JyX01pY3JvYmVfUGFwZXJfMjAyMS9STkEvaW5wdXQvMjAxNzA4MzAtc2FtcGxlcy9rYWxsaXN0b19vdXRfVjMuMApnemlwICovYWJ1bmRhbmNlLnRzdgpgYGAKCgojIyBHZXQgY291bnRzIGludG8gUgoKYGBge3J9CmxpYnJhcnkodGlkeXZlcnNlKQpsaWJyYXJ5KHN0cmluZ3IpCmxpYnJhcnkoZWRnZVIpCmBgYAoKCgpgYGB7cn0Ka2FsbGlzdG9fZmlsZXMgPC0gZGlyKHBhdGggPSAiLi4vLi4vaW5wdXQvMjAxNzA4MzAtc2FtcGxlcy9rYWxsaXN0b19vdXRfVjMuMCIscGF0dGVybj0iYWJ1bmRhbmNlLnRzdiIscmVjdXJzaXZlID0gVFJVRSxmdWxsLm5hbWVzID0gVFJVRSkKa2FsbGlzdG9fbmFtZXMgPC0gc3RyX3NwbGl0KGthbGxpc3RvX2ZpbGVzLCIvIixzaW1wbGlmeT1UUlVFKVssNl0KaGVhZChrYWxsaXN0b19uYW1lcykKYGBgCgpgYGB7cn0KY291bnRzIDwtIHRpYmJsZShzYW1wbGUgPSBrYWxsaXN0b19uYW1lcywgZmlsZSA9IGthbGxpc3RvX2ZpbGVzKSAlPiUKICBtdXRhdGUoY291bnRkYXRhID0gbWFwKGthbGxpc3RvX2ZpbGVzLCByZWFkX3RzdikpICU+JQogIHNlbGVjdCgtZmlsZSkKCmhlYWQoY291bnRzKQpgYGAKCnJlZm9ybWF0IGludG8gcm93cyA9IGdlbmVzIGFuZCBjb2x1bW5zID0gc2FtcGxlcwpgYGB7cn0KY291bnRzIDwtIGNvdW50cyAlPiUgdW5uZXN0KGNvdW50ZGF0YSkgJT4lCiAgbXV0YXRlKHNhbXBsZSA9IHN0cl9yZW1vdmUoc2FtcGxlLCAiX0wuKiIpKSAlPiUgCiAgc2VsZWN0KHNhbXBsZSwgdGFyZ2V0X2lkLCBlc3RfY291bnRzKSAlPiUKICBncm91cF9ieShzYW1wbGUsIHRhcmdldF9pZCkgJT4lCiAgc3VtbWFyaXplKGVzdF9jb3VudHM9c3VtKGVzdF9jb3VudHMpKSAlPiUgIyBzdW0gdXAgY291bnRzIGZyb20gbXVsdGlwbGUgbGFuZXMKICB1bmdyb3VwKCkgJT4lCiAgcGl2b3Rfd2lkZXIoaWRfY29scyA9IHRhcmdldF9pZCwKICAgICAgICAgICAgICBuYW1lc19mcm9tID0gc2FtcGxlLAogICAgICAgICAgICAgIHZhbHVlc19mcm9tID0gZXN0X2NvdW50cykKYGBgCgpgYGB7cn0KaGVhZChjb3VudHMpCmRpbShjb3VudHMpCmBgYAoKYGBge3J9CndyaXRlX2Nzdihjb3VudHMsIi4uLy4uL291dHB1dC8yMDE3MDgzMF9WMy4wX3Jhd19jb3VudHNfLmNzdi5neiIpCmBgYAoKCiMjIG1ha2Ugc2FtcGxlIGRlc2NyaXB0aW9uIGRhdGEgZnJhbWUKCgpgYGB7cn0Kc2FtcGxlLmRlc2NyaXB0aW9uIDwtIHRpYmJsZShzYW1wbGU9Y29sbmFtZXMoY291bnRzKVstMV0pICU+JQogIHNlcGFyYXRlKHNhbXBsZSwKICAgICAgICAgICBjKCJsb2NhdGlvbiIsInRpc3N1ZSIsImdlbm90eXBlIiwiYmxvY2siLCJwb3QiKSwKICAgICAgICAgICByZW1vdmU9RkFMU0UsCiAgICAgICAgICAgY29udmVydD1UUlVFKSAKaGVhZChzYW1wbGUuZGVzY3JpcHRpb24pCgojIyBnZXQgYWRkaXRpb25hbCBtZXRhZGF0YQpzYW1wbGUuaW5mbyA8LSByZWFkeGw6OnJlYWRfZXhjZWwoIi4uL2lucHV0L3d5MDAxX1JOQXNlcS54bHN4IixzaGVldCA9IDEpCmhlYWQoc2FtcGxlLmluZm8pCgojI2NvbWJpbmUKc2FtcGxlLmRlc2NyaXB0aW9uIDwtIGxlZnRfam9pbihzYW1wbGUuZGVzY3JpcHRpb24sIHNhbXBsZS5pbmZvKQpzYW1wbGUuZGVzY3JpcHRpb24gPC0gc2FtcGxlLmRlc2NyaXB0aW9uICU+JSAKICBtdXRhdGUoZ3JvdXA9cGFzdGUodGlzc3VlLGdlbm90eXBlLHNvaWwsYXV0b2NsYXZlLHNlcD0iXyIpKQpoZWFkKHNhbXBsZS5kZXNjcmlwdGlvbikKc2FtcGxlLmRlc2NyaXB0aW9uICU+JSBzdW1tYXJpemUobl9kaXN0aW5jdChncm91cCkpCmBgYAoKIyMgc3VtbWFyaXplIGNvdW50cwpgYGB7cn0KcGwub3JpZyA8LSBjb3VudHNbLC0xXSAlPiUgY29sU3VtcygpICU+JSB0aWJibGUoc2FtcGxlPW5hbWVzKC4pLGNvdW50PS4pICU+JQogIGdncGxvdChhZXMoeD1zYW1wbGUseT1jb3VudCkpICsgCiAgZ2VvbV9jb2woKSArCiAgdGhlbWUoYXhpcy50ZXh0LnggID0gZWxlbWVudF90ZXh0KGFuZ2xlPTkwLCB2anVzdD0wLjUsc2l6ZSA9IDcpKSAKcGwub3JpZwpgYGAKCiMjIGxvYWQgdG8gZWRnZVIsIG5vcm1hbGl6ZQoKYGBge3J9CiNjb25maXJtIHRoYXQgZXZlcnRoaW5nIGlzIGluIHRoZSByaWdodCBvcmRlcgphbGwoY29sbmFtZXMoY291bnRzKVstMV09PXNhbXBsZS5kZXNjcmlwdGlvbiRzYW1wbGUpCmRnZSA8LSBER0VMaXN0KGNvdW50c1ssLTFdLAogICAgICAgICAgICAgICBncm91cD1zYW1wbGUuZGVzY3JpcHRpb24kZ3JvdXAsCiAgICAgICAgICAgICAgIHNhbXBsZXM9c2FtcGxlLmRlc2NyaXB0aW9uLAogICAgICAgICAgICAgICBnZW5lcz1jb3VudHMkdGFyZ2V0X2lkKQpgYGAKCmBgYHtyfQpkZ2UgPC0gY2FsY05vcm1GYWN0b3JzKGRnZSkKYmFycGxvdChkZ2Ukc2FtcGxlcyRsaWIuc2l6ZSkKZ2dwbG90KGRnZSRzYW1wbGVzLGFlcyh4PXNhbXBsZSx5PW5vcm0uZmFjdG9ycyxmaWxsPXRpc3N1ZSkpICsgZ2VvbV9jb2woKSArIAogIHRoZW1lKGF4aXMudGV4dC54ICA9IGVsZW1lbnRfdGV4dChhbmdsZT05MCwgdmp1c3Q9MC41LHNpemUgPSA3KSkgCmdncGxvdChkZ2Ukc2FtcGxlcyxhZXMoeD1zYW1wbGUseT1ub3JtLmZhY3RvcnMsZmlsbD1nZW5vdHlwZSkpICsgZ2VvbV9jb2woKSArIAogIHRoZW1lKGF4aXMudGV4dC54ICA9IGVsZW1lbnRfdGV4dChhbmdsZT05MCwgdmp1c3Q9MC41LHNpemUgPSA3KSkgCmdncGxvdChkZ2Ukc2FtcGxlcyxhZXMoeD1zYW1wbGUseT1ub3JtLmZhY3RvcnMsZmlsbD1hcy5mYWN0b3IoYmxvY2spKSkgKyBnZW9tX2NvbCgpICsKICB0aGVtZShheGlzLnRleHQueCAgPSBlbGVtZW50X3RleHQoYW5nbGU9OTAsIHZqdXN0PTAuNSxzaXplID0gNykpIApgYGAKCkxvb2tzIGxpa2Ugd2Ugc2hvdWxkIG5vcm1hbGl6ZSBzZXBhcmF0ZWx5IGZvciByb290IGFuZCBsZWFmCgojIGRvIHNlcGFyYXRlbHkgZm9yIGxlYWYgYW5kIHJvb3QgdmFsdWVzCgpgYGB7cn0KY291bnRzLmxlYWYgPC0gY291bnRzICU+JSBzZWxlY3QodGFyZ2V0X2lkLCBjb250YWlucygibGVhZiIpKQpjb3VudHMucm9vdCA8LSBjb3VudHMgJT4lIHNlbGVjdCh0YXJnZXRfaWQsIGNvbnRhaW5zKCJyb290IikpCnNhbXBsZS5kZXNjcmlwdGlvbi5sZWFmIDwtIHNhbXBsZS5kZXNjcmlwdGlvbiAlPiUgZmlsdGVyKHRpc3N1ZT09ImxlYWYiKQpzYW1wbGUuZGVzY3JpcHRpb24ucm9vdCA8LSBzYW1wbGUuZGVzY3JpcHRpb24gJT4lIGZpbHRlcih0aXNzdWU9PSJyb290IikKYGBgCgpMZWFmCmBgYHtyfQojY29uZmlybSB0aGF0IGV2ZXJ0aGluZyBpcyBpbiB0aGUgcmlnaHQgb3JkZXIKYWxsKGNvbG5hbWVzKGNvdW50cy5sZWFmKVstMV09PXNhbXBsZS5kZXNjcmlwdGlvbi5sZWFmJHNhbXBsZSkKZGdlLmxlYWYgPC0gREdFTGlzdChjb3VudHMubGVhZlssLTFdLAogICAgICAgICAgICAgICAgICAgIGdyb3VwPXNhbXBsZS5kZXNjcmlwdGlvbi5sZWFmJGdyb3VwLAogICAgICAgICAgICAgICAgICAgIHNhbXBsZXM9c2FtcGxlLmRlc2NyaXB0aW9uLmxlYWYsCiAgICAgICAgICAgICAgICAgICAgZ2VuZXM9Y291bnRzLmxlYWYkdGFyZ2V0X2lkKQpkZ2UubGVhZiA8LSBjYWxjTm9ybUZhY3RvcnMoZGdlLmxlYWYpCmBgYAoKUm9vdApgYGB7cn0KI2NvbmZpcm0gdGhhdCBldmVydGhpbmcgaXMgaW4gdGhlIHJpZ2h0IG9yZGVyCmFsbChjb2xuYW1lcyhjb3VudHMucm9vdClbLTFdPT1zYW1wbGUuZGVzY3JpcHRpb24ucm9vdCRzYW1wbGUpCmRnZS5yb290IDwtIERHRUxpc3QoY291bnRzLnJvb3RbLC0xXSwKICAgICAgICAgICAgICAgICAgICBncm91cD1zYW1wbGUuZGVzY3JpcHRpb24ucm9vdCRncm91cCwKICAgICAgICAgICAgICAgICAgICBzYW1wbGVzPXNhbXBsZS5kZXNjcmlwdGlvbi5yb290LAogICAgICAgICAgICAgICAgICAgIGdlbmVzPWNvdW50cy5yb290JHRhcmdldF9pZCkKZGdlLnJvb3QgPC0gY2FsY05vcm1GYWN0b3JzKGRnZS5yb290KQpgYGAKCmBgYHtyfQpzYXZlKGRnZS5sZWFmLGRnZS5yb290LHNhbXBsZS5kZXNjcmlwdGlvbi5sZWFmLHNhbXBsZS5kZXNjcmlwdGlvbi5yb290LGZpbGU9Ii4uL291dHB1dC9lZGdlUl9kZ2Vfb2JqZWN0cy5SZGF0YSIpCmBgYAoKCiMjIFdyaXRlIG91dCBjcG0gdmFsdWVzCgpgYGB7cn0KY3BtLmxlYWYudyA8LSBiaW5kX2NvbHMoZGdlLmxlYWYkZ2VuZSxhc190aWJibGUoY3BtKGRnZS5sZWFmKSkpICU+JSBhc190aWJibGUoKSAlPiUgcmVuYW1lKHRyYW5zY3JpcHRfSUQ9Z2VuZXMpCmhlYWQoY3BtLmxlYWYudykKd3JpdGVfY3N2KGNwbS5sZWFmLncsIi4uL291dHB1dC9jcG1fd2lkZV8yMDE3MDYxN19sZWFmX3NhbXBsZXMuY3N2Lmd6IikKYGBgCgpgYGB7cn0KY3BtLnJvb3QudyA8LSBiaW5kX2NvbHMoZGdlLnJvb3QkZ2VuZSxhc190aWJibGUoY3BtKGRnZS5yb290KSkpICU+JSBhc190aWJibGUoKSAlPiUgcmVuYW1lKHRyYW5zY3JpcHRfSUQ9Z2VuZXMpCmhlYWQoY3BtLnJvb3QudykKd3JpdGVfY3N2KGNwbS5yb290LncsIi4uL291dHB1dC9jcG1fd2lkZV8yMDE3MDYxN19yb290X3NhbXBsZXMuY3N2Lmd6IikKYGBgCgoKQWxzbyBsZXQncyByZWZvcm1hdCB0aGlzIHRvIGxvbmcgZm9ybWF0IGFuZCBhZGQgbWV0YWRhdGEKCmBgYHtyfQpjcG0ubGVhZi5sb25nIDwtIGNwbS5sZWFmLncgJT4lIAogIGdhdGhlcigtdHJhbnNjcmlwdF9JRCxrZXkgPSBzYW1wbGUsdmFsdWU9Y3BtKSAlPiUKICBsZWZ0X2pvaW4oc2FtcGxlLmRlc2NyaXB0aW9uLmxlYWYpCmhlYWQoY3BtLmxlYWYubG9uZykKd3JpdGVfY3N2KGNwbS5sZWFmLmxvbmcsIi4uL291dHB1dC9jcG1fbG9uZ193aXRoX21ldGFkYXRhXzIwMTcwNjE3X2xlYWZfc2FtcGxlcy5jc3YuZ3oiKQpgYGAKCmBgYHtyfQpjcG0ucm9vdC5sb25nIDwtIGNwbS5yb290LncgJT4lIAogIGdhdGhlcigtdHJhbnNjcmlwdF9JRCxrZXkgPSBzYW1wbGUsdmFsdWU9Y3BtKSAlPiUKICBsZWZ0X2pvaW4oc2FtcGxlLmRlc2NyaXB0aW9uLnJvb3QpCmhlYWQoY3BtLnJvb3QubG9uZykKd3JpdGVfY3N2KGNwbS5yb290LmxvbmcsIi4uL291dHB1dC9jcG1fbG9uZ193aXRoX21ldGFkYXRhXzIwMTcwNjE3X3Jvb3Rfc2FtcGxlcy5jc3YuZ3oiKQpgYGAKCgojIyBjb21wdXRlIGFuZCB3cml0ZSBvdXQgdm9vbS10cmFuc2Zvcm1lZCB2YWx1ZXMKCmBgYHtyfQpkZXNpZ24ubGVhZiA8LSBtb2RlbC5tYXRyaXgofiBzYW1wbGUuZGVzY3JpcHRpb24ubGVhZiRncm91cCkKZGdlNHZvb20ubGVhZiA8LSBkZ2UubGVhZltyb3dTdW1zKGNwbShkZ2UubGVhZik+MSkgPj0gNiwsa2VlcC5saWIuc2l6ZXM9RkFMU0VdCmRnZTR2b29tLmxlYWYgPC0gY2FsY05vcm1GYWN0b3JzKGRnZTR2b29tLmxlYWYpCmRhdGEudm9vbS5sZWFmIDwtIHZvb20oZGdlNHZvb20ubGVhZixkZXNpZ24gPSBkZXNpZ24ubGVhZikKZGF0YS52b29tLmV4cC5sZWFmIDwtIGJpbmRfY29scyhkYXRhLnZvb20ubGVhZiRnZW5lcyxhc190aWJibGUoZGF0YS52b29tLmxlYWYkRSkpICU+JQogIHJlbmFtZSh0cmFuc2NyaXB0X0lEPWdlbmVzKSAlPiUgYXNfdGliYmxlKCkKaGVhZChkYXRhLnZvb20uZXhwLmxlYWYpCndyaXRlX2NzdihkYXRhLnZvb20uZXhwLmxlYWYsICIuLi9vdXRwdXQvdm9vbV9leHByZXNzaW9uXzIwMTcwNjE3X1Q2X2xlYWZfc2FtcGxlcy5jc3YuZ3oiKQpgYGAKCgpgYGB7cn0Kdm9vbS5sb25nLmxlYWYgPC0gZGF0YS52b29tLmV4cC5sZWFmICU+JSAKICBnYXRoZXIoLXRyYW5zY3JpcHRfSUQsa2V5ID0gc2FtcGxlLHZhbHVlPWV4cHJlc3Npb24pICU+JQogIGxlZnRfam9pbihzYW1wbGUuZGVzY3JpcHRpb24ubGVhZikKaGVhZCh2b29tLmxvbmcubGVhZikKaGlzdCh2b29tLmxvbmcubGVhZiRleHByZXNzaW9uKQp3cml0ZV9jc3Yodm9vbS5sb25nLmxlYWYsIi4uL291dHB1dC92b29tX2xvbmdfd2l0aF9tZXRhZGF0YV8yMDE3MDYxN19UNl9sZWFmX3NhbXBsZXMuY3N2Lmd6IikKYGBgCgoKYGBge3J9CmRlc2lnbi5yb290IDwtIG1vZGVsLm1hdHJpeCh+IHNhbXBsZS5kZXNjcmlwdGlvbi5yb290JGdyb3VwKQpkZ2U0dm9vbS5yb290IDwtIGRnZS5yb290W3Jvd1N1bXMoY3BtKGRnZS5yb290KT4xKSA+PSA2LCxrZWVwLmxpYi5zaXplcz1GQUxTRV0KZGdlNHZvb20ucm9vdCA8LSBjYWxjTm9ybUZhY3RvcnMoZGdlNHZvb20ucm9vdCkKZGF0YS52b29tLnJvb3QgPC0gdm9vbShkZ2U0dm9vbS5yb290LGRlc2lnbiA9IGRlc2lnbi5yb290KQpkYXRhLnZvb20uZXhwLnJvb3QgPC0gYmluZF9jb2xzKGRhdGEudm9vbS5yb290JGdlbmVzLGFzX3RpYmJsZShkYXRhLnZvb20ucm9vdCRFKSkgJT4lCiAgcmVuYW1lKHRyYW5zY3JpcHRfSUQ9Z2VuZXMpICU+JSBhc190aWJibGUoKQpoZWFkKGRhdGEudm9vbS5leHAucm9vdCkKd3JpdGVfY3N2KGRhdGEudm9vbS5leHAucm9vdCwgIi4uL291dHB1dC92b29tX2V4cHJlc3Npb25fMjAxNzA2MTdfVDZfcm9vdF9zYW1wbGVzLmNzdi5neiIpCmBgYAoKCmBgYHtyfQp2b29tLmxvbmcucm9vdCA8LSBkYXRhLnZvb20uZXhwLnJvb3QgJT4lIAogIGdhdGhlcigtdHJhbnNjcmlwdF9JRCxrZXkgPSBzYW1wbGUsdmFsdWU9ZXhwcmVzc2lvbikgJT4lCiAgbGVmdF9qb2luKHNhbXBsZS5kZXNjcmlwdGlvbi5yb290KQpoZWFkKHZvb20ubG9uZy5yb290KQpoaXN0KHZvb20ubG9uZy5yb290JGV4cHJlc3Npb24pCndyaXRlX2Nzdih2b29tLmxvbmcucm9vdCwiLi4vb3V0cHV0L3Zvb21fbG9uZ193aXRoX21ldGFkYXRhXzIwMTcwNjE3X1Q2X3Jvb3Rfc2FtcGxlcy5jc3YuZ3oiKQpgYGAKCndyaXRlIGl0IHRvIGlyb2RzCgpOZWVkIHRvIHJ1biB0aGlzIHlvdXJzZWxmIGluIHRlcm1pbmFsCgpgYGB7ciwgZW5naW5lPSdiYXNoJywgZXZhbD1GQUxTRX0KaWluaXQKaWNkIC9pcGxhbnQvaG9tZS9zaGFyZWQvdWNkLmJyYXNzaWNhL2FuYWx5c2VzL0JyYXBhX01pY3JvYmlvbWVfUk5Bc2VxLwpmb3IgZiBpbiAobHMgY3BtKikKICAgIGVjaG8gJGYKICAgIGlwdXQgLXZmICRmCmVuZApmb3IgZiBpbiAobHMgdm9vbSopCiAgICBlY2hvICRmCiAgICBpcHV0IC12ZiAkZgplbmQKYGBgCgojIyByZWFkIGRpc3RyaWJ1dGlvbgoKTWlrZSBhc2tlZCBpZiB0aGUgZGlmZmVyZW5jZSBpbiBub3JtYWxpemF0aW9uIGZhY3RvcnMgaW4gbGVhZnMgdnMgcm9vdHMgd2FzIGR1ZSB0byBoaWdoIGFidW5kYW5jZSBvZiBwaG90b3N5bnRoZXNpcyB0cmFuc2NyaXB0cyBpbiBsZWFmcy4gIChBbHRob3VnaCBsZWFmcyBoYXZlIGxvd2VyIG5vcm1hbGl6YXRpb24gZmFjdG9yKQoKU3RhcnQgYnkgZG9pbmcgYSBzaW1wbGlzdGljIG5vcm1hbGl6YXRpb24ganVzdCBieSBsaWJyYXJ5IHNpemUuICBUaGVuIGxvb2sgYXQgZGlzdHJpYnV0aW9uIG9mIG1vc3QgYWJ1bmRhbnQgY291bnRzLgoKYGBge3J9CmNvdW50cy5sZWFmLm5vcm0gPC0gY291bnRzLmxlYWYgJT4lIAogIG11dGF0ZV9hdCgtMSxmdW5zKC4vc3VtKC4pKSkgJT4lIAogIHRyYW5zbXV0ZSh0YXJnZXRfaWQ9dGFyZ2V0X2lkLG1lYW49IHtzZWxlY3QoLiwtdGFyZ2V0X2lkKSAlPiUgcm93TWVhbnMoKX0pICU+JQogIGFycmFuZ2UoZGVzYyhtZWFuKSkgJT4lIG11dGF0ZShjdW1zdW09Y3Vtc3VtKG1lYW4pLHJhbms9cm93X251bWJlcigpLHNhbXBsZT0ibGVhZiIpCmNvdW50cy5sZWFmLm5vcm0KYGBgCmBgYHtyfQpjb3VudHMucm9vdC5ub3JtIDwtIGNvdW50cy5yb290ICU+JSAKICBtdXRhdGVfYXQoLTEsZnVucyguL3N1bSguKSkpICU+JSAKICB0cmFuc211dGUodGFyZ2V0X2lkPXRhcmdldF9pZCxtZWFuPSB7c2VsZWN0KC4sLXRhcmdldF9pZCkgJT4lIHJvd01lYW5zKCl9KSAlPiUKICBhcnJhbmdlKGRlc2MobWVhbikpICU+JSBtdXRhdGUoY3Vtc3VtPWN1bXN1bShtZWFuKSxyYW5rPXJvd19udW1iZXIoKSxzYW1wbGU9InJvb3QiKQpjb3VudHMucm9vdC5ub3JtCmBgYAoKYGBge3J9CnJiaW5kKGNvdW50cy5sZWFmLm5vcm0sY291bnRzLnJvb3Qubm9ybSkgJT4lCiAgZ2dwbG90KGFlcyh4PXJhbmsseT1jdW1zdW0sY29sb3I9c2FtcGxlKSkgKwogIGdlb21fbGluZSgpICt4bGltKDAsMjAwMDApCmBgYAoKYGBge3J9CnJiaW5kKGNvdW50cy5sZWFmLm5vcm0sY291bnRzLnJvb3Qubm9ybSkgJT4lCiAgZ2dwbG90KGFlcyh4PXJhbmsseT1tZWFuLGNvbG9yPXNhbXBsZSkpICsKICBnZW9tX2xpbmUoKSAgKyBzY2FsZV95X2xvZzEwKCkKYGBgCgpgYGB7cn0KcmJpbmQoY291bnRzLmxlYWYubm9ybSxjb3VudHMucm9vdC5ub3JtKSAlPiUgZmlsdGVyKHJhbmsgPCA0MSkgJT4lCiAgZ2dwbG90KGFlcyh4PXJhbmsseT1tZWFuLGZpbGw9c2FtcGxlKSkgKwogIGdlb21fY29sKHBvc2l0aW9uID0gImRvZGdlIikgCmBgYAoKYGBge3J9CmFubm90YXRpb24gPC0gcmVhZF9jc3YoIi4uLy4uLy4uL0Fubm90YXRpb24vb3V0cHV0L3YzLjBhbm5vdGF0aW9uL0JyYXBhX1YzLjBfYW5ub3RhdGVkLmNzdiIpCmBgYAoKYGBge3J9CnRvcC5leHByZXNzZWQubGVhZiA8LSBjb3VudHMubGVhZi5ub3JtICU+JSBmaWx0ZXIocmFuazwyMSkgJT4lIGxlZnRfam9pbihhbm5vdGF0aW9uLGJ5PWMoInRhcmdldF9pZCI9Im5hbWUiKSkgJT4lIHNlbGVjdCgidGFyZ2V0X2lkIiwibWVhbiIsInJhbmsiLCJBR0kiLCJBdF9zeW1ib2wiLCJBdF9kZXNjcmlwdGlvbiIpICU+JSBhcnJhbmdlKHJhbmspCnRvcC5leHByZXNzZWQubGVhZgp3cml0ZV9jc3YodG9wLmV4cHJlc3NlZC5sZWFmLCIuLi9vdXRwdXQvdG9wLmV4cHJlc3NlZC5sZWFmLmNzdiIpCmBgYAoKYGBge3J9CnRvcC5leHByZXNzZWQucm9vdCA8LSBjb3VudHMucm9vdC5ub3JtICU+JSBmaWx0ZXIocmFuazwyMSkgJT4lIGxlZnRfam9pbihhbm5vdGF0aW9uLGJ5PWMoInRhcmdldF9pZCI9Im5hbWUiKSkgJT4lIHNlbGVjdCgidGFyZ2V0X2lkIiwibWVhbiIsInJhbmsiLCJBR0kiLCJBdF9zeW1ib2wiLCJBdF9kZXNjcmlwdGlvbiIpICU+JSBhcnJhbmdlKHJhbmspCnRvcC5leHByZXNzZWQucm9vdAp3cml0ZV9jc3YodG9wLmV4cHJlc3NlZC5yb290LCIuLi9vdXRwdXQvdG9wLmV4cHJlc3NlZC5yb290LmNzdiIpCmBgYA==